<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>How transform checkpoints work | Elasticsearch Guide [7.7] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch Guide [7.7]">
<link rel="up" href="transforms.html" title="Transforming data">
<link rel="prev" href="transform-usage.html" title="When to use transforms">
<link rel="next" href="transform-api-quickref.html" title="API quick reference">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Reference/7.7">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="7.7">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch Guide [7.7]</a></span>
»
<span class="breadcrumb-link"><a href="data-rollup-transform.html">Roll up or transform your data</a></span>
»
<span class="breadcrumb-link"><a href="transforms.html">Transforming data</a></span>
»
<span class="breadcrumb-node">How transform checkpoints work</span>
</div>
<div class="navheader">
<span class="prev">
<a href="transform-usage.html">« When to use transforms</a>
</span>
<span class="next">
<a href="transform-api-quickref.html">API quick reference »</a>
</span>
</div>
<div class="section xpack">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="transform-checkpoints"></a>How transform checkpoints work<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/checkpoints.asciidoc">edit</a><a class="xpack_tag" href="/subscriptions"></a>
</h2>
</div></div></div>

<p>Each time a transform examines the source indices and creates or
updates the destination index, it generates a <em>checkpoint</em>.</p>
<p>If your transform runs only once, there is logically only one
checkpoint. If your transform runs continuously, however, it creates
checkpoints as it ingests and transforms new source data.</p>
<p>To create a checkpoint, the continuous transform:</p>
<div class="olist orderedlist">
<ol class="orderedlist">
<li class="listitem">
<p>Checks for changes to source indices.</p>
<p>Using a simple periodic timer, the transform checks for changes to
the source indices. This check is done based on the interval defined in the
transform’s <code class="literal">frequency</code> property.</p>
<p>If the source indices remain unchanged or if a checkpoint is already in progress
then it waits for the next timer.</p>
</li>
<li class="listitem">
<p>Identifies which entities have changed.</p>
<p>The transform searches to see which entities have changed since the
last time it checked. The <code class="literal">sync</code> configuration object in the transform
identifies a time field in the source indices. The transform uses the values
in that field to synchronize the source and destination indices.</p>
</li>
<li class="listitem">
<p>Updates the destination index (the data frame) with the changed entities.</p>
<p>The transform applies changes related to either new or changed
entities to the destination index. The set of changed entities is paginated. For
each page, the transform performs a composite aggregation using a
<code class="literal">terms</code> query. After all the pages of changes have been applied, the checkpoint
is complete.</p>
</li>
</ol>
</div>
<p>This checkpoint process involves both search and indexing activity on the
cluster. We have attempted to favor control over performance while developing
transforms. We decided it was preferable for the
transform to take longer to complete, rather than to finish quickly
and take precedence in resource consumption. That being said, the cluster still
requires enough resources to support both the composite aggregation search and
the indexing of its results.</p>
<div class="tip admon">
<div class="icon"></div>
<div class="admon_content">
<p>If the cluster experiences unsuitable performance degradation due to the
transform, stop the transform. Consider whether you can apply a
source query to the transform to reduce the scope of data it
processes. Also consider whether the cluster has sufficient resources in place
to support both the composite aggregation search and the indexing of its
results.</p>
</div>
</div>
<h4>
<a id="ml-transform-checkpoint-errors"></a>Error handling<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/checkpoints.asciidoc">edit</a>
</h4>
<p>Failures in transforms tend to be related to searching or indexing.
To increase the resiliency of transforms, the cursor positions of
the aggregated search and the changed entities search are tracked in memory and
persisted periodically.</p>
<p>Checkpoint failures can be categorized as follows:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
Temporary failures: The checkpoint is retried. If 10 consecutive failures
occur, the transform has a failed status. For example, this
situation might occur when there are shard failures and queries return only
partial results.
</li>
<li class="listitem">
Irrecoverable failures: The transform immediately fails. For
example, this situation occurs when the source index is not found.
</li>
<li class="listitem">
Adjustment failures: The transform retries with adjusted settings.
For example, if a parent circuit breaker memory errors occur during the
composite aggregation, the transform receives partial results. The aggregated
search is retried with a smaller number of buckets. This retry is performed at
the interval defined in the <code class="literal">frequency</code> property for the transform. If the
search is retried to the point where it reaches a minimal number of buckets, an
irrecoverable failure occurs.
</li>
</ul>
</div>
<p>If the node running the transforms fails, the transform restarts
from the most recent persisted cursor position. This recovery process might
repeat some of the work the transform had already done, but it ensures data
consistency.</p>
</div>
<div class="navfooter">
<span class="prev">
<a href="transform-usage.html">« When to use transforms</a>
</span>
<span class="next">
<a href="transform-api-quickref.html">API quick reference »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
